x86: vMSI: Fix msi irq affinity issue for hvm guest.

There is a race between guest setting new vector and doing EOI on old
vector.  Once guest sets new vector before its doing EOI on vector,
when guest does eoi, hypervisor may fail to find the related pirq, and
hypervisor may miss to EOI real vector and leads to system hang.  We
may need to add a timer for each pirq interrupt source to avoid host
hang, but this is another topic, and will be addressed later.

Signed-off-by: Xiantao Zhang <xiantao.zhang@intel.com>

diff --git a/xen/arch/x86/hvm/vmsi.c b/xen/arch/x86/hvm/vmsi.c
index e312c442d7bc6fb8754c401506856383e7523881..2c2b7b9b29418207c11ab9a3efe2f02f48056a92 100644 (file)
--- a/xen/arch/x86/hvm/vmsi.c
+++ b/xen/arch/x86/hvm/vmsi.c
@@ -92,8 +92,11 @@ int vmsi_deliver(struct domain *d, int pirq)
     case dest_LowestPrio:
     {
         target = vlapic_lowest_prio(d, NULL, 0, dest, dest_mode);
-        if ( target != NULL )
+        if ( target != NULL ) {
             vmsi_inj_irq(d, target, vector, trig_mode, delivery_mode);
+            hvm_irq_dpci->mirq[pirq].gmsi.old_gvec =
+                                    hvm_irq_dpci->mirq[pirq].gmsi.gvec;
+        }
         else
             HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "null round robin: "
                         "vector=%x delivery_mode=%x\n",
@@ -106,9 +109,12 @@ int vmsi_deliver(struct domain *d, int pirq)
     {
         for_each_vcpu ( d, v )
             if ( vlapic_match_dest(vcpu_vlapic(v), NULL,
-                                   0, dest, dest_mode) )
+                                   0, dest, dest_mode) ) {
                 vmsi_inj_irq(d, vcpu_vlapic(v),
                              vector, trig_mode, delivery_mode);
+                hvm_irq_dpci->mirq[pirq].gmsi.old_gvec =
+                                    hvm_irq_dpci->mirq[pirq].gmsi.gvec;
+            }
         break;
     }
 

diff --git a/xen/drivers/passthrough/io.c b/xen/drivers/passthrough/io.c
index 4e4c8b9ee05b1032b31246f56fae9df248665154..ce9f0884f29818ab48884b98a930115ffa9b1dea 100644 (file)
--- a/xen/drivers/passthrough/io.c
+++ b/xen/drivers/passthrough/io.c
@@ -164,7 +164,9 @@ int pt_irq_create_bind_vtd(
         {
             hvm_irq_dpci->mirq[pirq].flags = HVM_IRQ_DPCI_MACH_MSI |
                                              HVM_IRQ_DPCI_GUEST_MSI;
+            hvm_irq_dpci->mirq[pirq].gmsi.old_gvec = pt_irq_bind->u.msi.gvec;
             hvm_irq_dpci->mirq[pirq].gmsi.gvec = pt_irq_bind->u.msi.gvec;
+            hvm_irq_dpci->mirq[pirq].gmsi.old_gflags = pt_irq_bind->u.msi.gflags;
             hvm_irq_dpci->mirq[pirq].gmsi.gflags = pt_irq_bind->u.msi.gflags;
             /* bind after hvm_irq_dpci is setup to avoid race with irq handler*/
             rc = pirq_guest_bind(d->vcpu[0], pirq, 0);
@@ -178,6 +180,8 @@ int pt_irq_create_bind_vtd(
             {
                 hvm_irq_dpci->mirq[pirq].gmsi.gflags = 0;
                 hvm_irq_dpci->mirq[pirq].gmsi.gvec = 0;
+                hvm_irq_dpci->mirq[pirq].gmsi.old_gvec = 0;
+                hvm_irq_dpci->mirq[pirq].gmsi.old_gflags = 0;
                 hvm_irq_dpci->mirq[pirq].flags = 0;
                 clear_bit(pirq, hvm_irq_dpci->mapping);
                 spin_unlock(&d->event_lock);
@@ -195,8 +199,14 @@ int pt_irq_create_bind_vtd(
             }
  
             /* if pirq is already mapped as vmsi, update the guest data/addr */
-            hvm_irq_dpci->mirq[pirq].gmsi.gvec = pt_irq_bind->u.msi.gvec;
-            hvm_irq_dpci->mirq[pirq].gmsi.gflags = pt_irq_bind->u.msi.gflags;
+            if ( hvm_irq_dpci->mirq[pirq].gmsi.gvec != pt_irq_bind->u.msi.gvec ) {
+                hvm_irq_dpci->mirq[pirq].gmsi.old_gvec =
+                                    hvm_irq_dpci->mirq[pirq].gmsi.gvec;
+                hvm_irq_dpci->mirq[pirq].gmsi.old_gflags =
+                                    hvm_irq_dpci->mirq[pirq].gmsi.gflags;
+                hvm_irq_dpci->mirq[pirq].gmsi.gvec = pt_irq_bind->u.msi.gvec;
+                hvm_irq_dpci->mirq[pirq].gmsi.gflags = pt_irq_bind->u.msi.gflags;
+            }
         }
         /* Caculate dest_vcpu_id for MSI-type pirq migration */
         dest = hvm_irq_dpci->mirq[pirq].gmsi.gflags & VMSI_DEST_ID_MASK;
@@ -424,14 +434,21 @@ void hvm_dpci_msi_eoi(struct domain *d, int vector)
           pirq = find_next_bit(hvm_irq_dpci->mapping, d->nr_pirqs, pirq + 1) )
     {
         if ( (!(hvm_irq_dpci->mirq[pirq].flags & HVM_IRQ_DPCI_MACH_MSI)) ||
-                (hvm_irq_dpci->mirq[pirq].gmsi.gvec != vector) )
+                (hvm_irq_dpci->mirq[pirq].gmsi.gvec != vector &&
+                 hvm_irq_dpci->mirq[pirq].gmsi.old_gvec != vector) )
             continue;
 
-        dest = hvm_irq_dpci->mirq[pirq].gmsi.gflags & VMSI_DEST_ID_MASK;
-        dest_mode = !!(hvm_irq_dpci->mirq[pirq].gmsi.gflags & VMSI_DM_MASK);
+        if ( hvm_irq_dpci->mirq[pirq].gmsi.gvec == vector ) {
+            dest = hvm_irq_dpci->mirq[pirq].gmsi.gflags & VMSI_DEST_ID_MASK;
+            dest_mode = !!(hvm_irq_dpci->mirq[pirq].gmsi.gflags & VMSI_DM_MASK);
+        } else {
+            dest = hvm_irq_dpci->mirq[pirq].gmsi.old_gflags & VMSI_DEST_ID_MASK;
+            dest_mode = !!(hvm_irq_dpci->mirq[pirq].gmsi.old_gflags & VMSI_DM_MASK);
+        }
         if ( vlapic_match_dest(vcpu_vlapic(current), NULL, 0, dest, dest_mode) )
             break;
     }
+
     if ( pirq < d->nr_pirqs )
         __msi_pirq_eoi(d, pirq);
     spin_unlock(&d->event_lock);

diff --git a/xen/include/xen/hvm/irq.h b/xen/include/xen/hvm/irq.h
index c1747ed73c6172891a5090c274a7b0b755e7be06..9e2eedbf6a7f10b99b71d0b3246ae94bdf47c676 100644 (file)
--- a/xen/include/xen/hvm/irq.h
+++ b/xen/include/xen/hvm/irq.h
@@ -58,8 +58,10 @@ struct dev_intx_gsi_link {
 #define GLFAGS_SHIFT_TRG_MODE       15
 
 struct hvm_gmsi_info {
-    uint32_t gvec;
+    uint16_t gvec;
+    uint16_t old_gvec;
     uint32_t gflags;
+    uint32_t old_gflags;
     int dest_vcpu_id; /* -1 :multi-dest, non-negative: dest_vcpu_id */
 };